{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d8ec632-1d75-4f1c-814c-459ad4e1ff89",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "# import torch\n",
    "# torch.cuda.set_device(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "69565362-a932-4be7-a083-5902e3c6d5d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, GPT2LMHeadModel, AutoConfig\n",
    "from transformers import GPT2Tokenizer,GPT2Model,AutoModel\n",
    "\n",
    "from transformers import DataCollatorForLanguageModeling\n",
    "from transformers import Trainer, TrainingArguments\n",
    "from transformers import LineByLineTextDataset\n",
    "from tokenizers import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4c982701-4078-4e2a-8976-2f335177295f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #然后我们可以使用from_file() 方法从该文件里重新加载 Tokenizer 对象：\n",
    "new_tokenizer = Tokenizer.from_file(\"tokenizer8.json\")\n",
    "# #或者下面方法\n",
    "# from transformers import GPT2TokenizerFast\n",
    "# tokenizer = GPT2TokenizerFast(tokenizer_object=new_tokenizer)\n",
    "\n",
    "from transformers import PreTrainedTokenizerFast\n",
    "\n",
    "tokenizer = PreTrainedTokenizerFast(\n",
    "    tokenizer_object=new_tokenizer,\n",
    "    bos_token=\"<s>\",\n",
    "    eos_token=\"</s>\",\n",
    "    unk_token=\"<unk>\",\n",
    "    pad_token=\"<pad>\",\n",
    "    cls_token=\"<cls>\",\n",
    "    sep_token=\"<sep>\",\n",
    "    mask_token=\"<mask>\",\n",
    "    padding_side=\"left\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "93fe531f-28be-4018-b62c-3a850688017a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#model = GPT2LMHeadModel.from_pretrained(\"gpt2\")\n",
    "context_length = 512\n",
    "config = AutoConfig.from_pretrained(\n",
    "    \"gpt2\",\n",
    "    vocab_size=len(tokenizer), #这里确定词的id纬度\n",
    "    n_ctx=context_length, #  Dimensionality of the causal mask (usually same as n_positions).default 1024\n",
    "    bos_token_id=tokenizer.bos_token_id,\n",
    "    eos_token_id=tokenizer.eos_token_id,\n",
    ")\n",
    "\n",
    "model = GPT2LMHeadModel(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9d708480-5d21-4990-9726-c3198c28a3ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 634318\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "dna_dataset = load_dataset(\"text\", data_files=\"human3.fna.line\")\n",
    "dna_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "24df9c8d-d88a-49a0-97fc-2e7c386ddd2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 570886\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 63432\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_train_devtest = dna_dataset['train'].train_test_split(test_size=0.1, seed=42)\n",
    "ds_train_devtest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "48307677-399f-4d8c-b67d-358b51f54bbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 570886\n",
       "    })\n",
       "    valid: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 63432\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datasets import load_dataset, DatasetDict\n",
    "\n",
    "raw_datasets = DatasetDict(\n",
    "    {\n",
    "        \"train\": ds_train_devtest[\"train\"],  # .shuffle().select(range(50000)),\n",
    "        \"valid\": ds_train_devtest[\"test\"],  # .shuffle().select(range(500))\n",
    "    }\n",
    ")\n",
    "raw_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "47005cd4-5070-425f-a3bd-b5d4fbd25f36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids'],\n",
       "        num_rows: 541337\n",
       "    })\n",
       "    valid: Dataset({\n",
       "        features: ['input_ids'],\n",
       "        num_rows: 60111\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def tokenize(element):\n",
    "    outputs = tokenizer(\n",
    "        element[\"text\"],\n",
    "        truncation=True,\n",
    "        max_length=context_length,\n",
    "        return_overflowing_tokens=True,\n",
    "        return_length=True,\n",
    "    )\n",
    "    input_batch = []\n",
    "    for length, input_ids in zip(outputs[\"length\"], outputs[\"input_ids\"]):\n",
    "        if length == context_length:\n",
    "            input_batch.append(input_ids) #不要padding，只要长度足够的\n",
    "    return {\"input_ids\": input_batch}\n",
    "\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(\n",
    "    tokenize, batched=True, remove_columns=raw_datasets[\"train\"].column_names\n",
    ")\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6dbb9082-ff92-4523-b20d-416f99de7e66",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [73,\n",
       "  107,\n",
       "  20547,\n",
       "  19867,\n",
       "  14306,\n",
       "  22312,\n",
       "  4412,\n",
       "  36978,\n",
       "  33592,\n",
       "  5468,\n",
       "  29037,\n",
       "  22001,\n",
       "  15135,\n",
       "  5025,\n",
       "  12427,\n",
       "  31382,\n",
       "  19551,\n",
       "  590,\n",
       "  16250,\n",
       "  203,\n",
       "  21008,\n",
       "  11141,\n",
       "  1028,\n",
       "  2153,\n",
       "  45467,\n",
       "  167,\n",
       "  21638,\n",
       "  490,\n",
       "  3626,\n",
       "  46856,\n",
       "  16632,\n",
       "  6603,\n",
       "  3940,\n",
       "  19247,\n",
       "  3880,\n",
       "  21002,\n",
       "  15365,\n",
       "  8824,\n",
       "  2271,\n",
       "  12868,\n",
       "  24938,\n",
       "  29905,\n",
       "  11964,\n",
       "  39709,\n",
       "  12863,\n",
       "  22195,\n",
       "  12588,\n",
       "  116,\n",
       "  35063,\n",
       "  20,\n",
       "  7790,\n",
       "  1040,\n",
       "  14252,\n",
       "  32395,\n",
       "  1234,\n",
       "  10215,\n",
       "  1782,\n",
       "  25701,\n",
       "  5631,\n",
       "  14778,\n",
       "  3227,\n",
       "  5135,\n",
       "  36054,\n",
       "  9635,\n",
       "  45091,\n",
       "  15549,\n",
       "  32,\n",
       "  4087,\n",
       "  35213,\n",
       "  32651,\n",
       "  425,\n",
       "  35970,\n",
       "  33822,\n",
       "  23929,\n",
       "  29644,\n",
       "  343,\n",
       "  41474,\n",
       "  25537,\n",
       "  18959,\n",
       "  129,\n",
       "  28308,\n",
       "  36009,\n",
       "  3986,\n",
       "  30262,\n",
       "  4326,\n",
       "  35175,\n",
       "  48905,\n",
       "  11467,\n",
       "  4792,\n",
       "  6366,\n",
       "  17781,\n",
       "  2703,\n",
       "  24981,\n",
       "  49026,\n",
       "  2009,\n",
       "  26687,\n",
       "  37821,\n",
       "  8902,\n",
       "  10946,\n",
       "  21718,\n",
       "  31230,\n",
       "  37402,\n",
       "  43040,\n",
       "  25200,\n",
       "  29235,\n",
       "  91,\n",
       "  26718,\n",
       "  17708,\n",
       "  30280,\n",
       "  15076,\n",
       "  4658,\n",
       "  10128,\n",
       "  7676,\n",
       "  27260,\n",
       "  6232,\n",
       "  1961,\n",
       "  27283,\n",
       "  15783,\n",
       "  14078,\n",
       "  5296,\n",
       "  38133,\n",
       "  3260,\n",
       "  4971,\n",
       "  5687,\n",
       "  132,\n",
       "  3571,\n",
       "  30031,\n",
       "  11240,\n",
       "  6930,\n",
       "  17522,\n",
       "  43321,\n",
       "  194,\n",
       "  21346,\n",
       "  11116,\n",
       "  1701,\n",
       "  13834,\n",
       "  3120,\n",
       "  2155,\n",
       "  25762,\n",
       "  23425,\n",
       "  12032,\n",
       "  30018,\n",
       "  265,\n",
       "  43463,\n",
       "  122,\n",
       "  5363,\n",
       "  3459,\n",
       "  24868,\n",
       "  11311,\n",
       "  15582,\n",
       "  34460,\n",
       "  31852,\n",
       "  21758,\n",
       "  19564,\n",
       "  12698,\n",
       "  6282,\n",
       "  15848,\n",
       "  5784,\n",
       "  41108,\n",
       "  34,\n",
       "  412,\n",
       "  25783,\n",
       "  12345,\n",
       "  1975,\n",
       "  47677,\n",
       "  10051,\n",
       "  37965,\n",
       "  35756,\n",
       "  10626,\n",
       "  29490,\n",
       "  19680,\n",
       "  1380,\n",
       "  17032,\n",
       "  1336,\n",
       "  18041,\n",
       "  24185,\n",
       "  5493,\n",
       "  25163,\n",
       "  2911,\n",
       "  12585,\n",
       "  3526,\n",
       "  43709,\n",
       "  2550,\n",
       "  4225,\n",
       "  476,\n",
       "  23351,\n",
       "  38474,\n",
       "  840,\n",
       "  418,\n",
       "  1372,\n",
       "  951,\n",
       "  1519,\n",
       "  31367,\n",
       "  3412,\n",
       "  7500,\n",
       "  2475,\n",
       "  48180,\n",
       "  846,\n",
       "  4797,\n",
       "  4608,\n",
       "  10312,\n",
       "  380,\n",
       "  43183,\n",
       "  18744,\n",
       "  37617,\n",
       "  14,\n",
       "  5062,\n",
       "  565,\n",
       "  5843,\n",
       "  6828,\n",
       "  6426,\n",
       "  4131,\n",
       "  45184,\n",
       "  589,\n",
       "  49312,\n",
       "  578,\n",
       "  13242,\n",
       "  19539,\n",
       "  11018,\n",
       "  6101,\n",
       "  39679,\n",
       "  47649,\n",
       "  17984,\n",
       "  553,\n",
       "  43228,\n",
       "  41369,\n",
       "  2376,\n",
       "  5355,\n",
       "  22638,\n",
       "  26909,\n",
       "  16121,\n",
       "  30942,\n",
       "  21417,\n",
       "  14352,\n",
       "  2787,\n",
       "  9389,\n",
       "  25953,\n",
       "  21621,\n",
       "  123,\n",
       "  28797,\n",
       "  14157,\n",
       "  23787,\n",
       "  1099,\n",
       "  3136,\n",
       "  15200,\n",
       "  1196,\n",
       "  6506,\n",
       "  5903,\n",
       "  7768,\n",
       "  6542,\n",
       "  36274,\n",
       "  11655,\n",
       "  18790,\n",
       "  1447,\n",
       "  20605,\n",
       "  37718,\n",
       "  3195,\n",
       "  46889,\n",
       "  1456,\n",
       "  36873,\n",
       "  2276,\n",
       "  42758,\n",
       "  41424,\n",
       "  5490,\n",
       "  289,\n",
       "  125,\n",
       "  13631,\n",
       "  47321,\n",
       "  35687,\n",
       "  32069,\n",
       "  35139,\n",
       "  9630,\n",
       "  13100,\n",
       "  10615,\n",
       "  40006,\n",
       "  8819,\n",
       "  17364,\n",
       "  5637,\n",
       "  16191,\n",
       "  43734,\n",
       "  30885,\n",
       "  3304,\n",
       "  28212,\n",
       "  2709,\n",
       "  7820,\n",
       "  49009,\n",
       "  6940,\n",
       "  1330,\n",
       "  26193,\n",
       "  27633,\n",
       "  2581,\n",
       "  36318,\n",
       "  29291,\n",
       "  2034,\n",
       "  3911,\n",
       "  21641,\n",
       "  242,\n",
       "  4356,\n",
       "  2071,\n",
       "  8363,\n",
       "  1966,\n",
       "  21903,\n",
       "  2577,\n",
       "  45163,\n",
       "  8049,\n",
       "  30353,\n",
       "  33292,\n",
       "  45557,\n",
       "  1819,\n",
       "  13217,\n",
       "  22381,\n",
       "  1504,\n",
       "  5494,\n",
       "  3883,\n",
       "  495,\n",
       "  15557,\n",
       "  2246,\n",
       "  8187,\n",
       "  12002,\n",
       "  9779,\n",
       "  10935,\n",
       "  6867,\n",
       "  1854,\n",
       "  29947,\n",
       "  4851,\n",
       "  2670,\n",
       "  830,\n",
       "  129,\n",
       "  16289,\n",
       "  550,\n",
       "  425,\n",
       "  10096,\n",
       "  14944,\n",
       "  5989,\n",
       "  23040,\n",
       "  3901,\n",
       "  1906,\n",
       "  46214,\n",
       "  268,\n",
       "  8436,\n",
       "  21268,\n",
       "  11088,\n",
       "  11860,\n",
       "  8253,\n",
       "  2678,\n",
       "  40037,\n",
       "  2495,\n",
       "  4370,\n",
       "  5487,\n",
       "  5378,\n",
       "  22467,\n",
       "  1061,\n",
       "  649,\n",
       "  41563,\n",
       "  8125,\n",
       "  49155,\n",
       "  1120,\n",
       "  26004,\n",
       "  6644,\n",
       "  5869,\n",
       "  908,\n",
       "  6962,\n",
       "  472,\n",
       "  31180,\n",
       "  23939,\n",
       "  18116,\n",
       "  459,\n",
       "  17812,\n",
       "  16010,\n",
       "  5314,\n",
       "  17738,\n",
       "  42276,\n",
       "  3814,\n",
       "  12386,\n",
       "  424,\n",
       "  21271,\n",
       "  3667,\n",
       "  29,\n",
       "  40268,\n",
       "  11648,\n",
       "  8796,\n",
       "  28030,\n",
       "  7523,\n",
       "  47066,\n",
       "  12271,\n",
       "  43677,\n",
       "  7861,\n",
       "  679,\n",
       "  29307,\n",
       "  1675,\n",
       "  27427,\n",
       "  9622,\n",
       "  7163,\n",
       "  11873,\n",
       "  1081,\n",
       "  33528,\n",
       "  4145,\n",
       "  175,\n",
       "  372,\n",
       "  1622,\n",
       "  33834,\n",
       "  308,\n",
       "  4177,\n",
       "  327,\n",
       "  30921,\n",
       "  12583,\n",
       "  18921,\n",
       "  2827,\n",
       "  49421,\n",
       "  218,\n",
       "  18358,\n",
       "  4204,\n",
       "  12254,\n",
       "  12611,\n",
       "  18940,\n",
       "  21340,\n",
       "  23723,\n",
       "  20679,\n",
       "  277,\n",
       "  16357,\n",
       "  19,\n",
       "  3189,\n",
       "  410,\n",
       "  410,\n",
       "  22327,\n",
       "  35561,\n",
       "  314,\n",
       "  11129,\n",
       "  38145,\n",
       "  12258,\n",
       "  3480,\n",
       "  1055,\n",
       "  6009,\n",
       "  91,\n",
       "  6504,\n",
       "  37695,\n",
       "  35549,\n",
       "  2912,\n",
       "  1296,\n",
       "  38,\n",
       "  2677,\n",
       "  4997,\n",
       "  17295,\n",
       "  6479,\n",
       "  21808,\n",
       "  10362,\n",
       "  33671,\n",
       "  996,\n",
       "  37517,\n",
       "  932,\n",
       "  29459,\n",
       "  13849,\n",
       "  6114,\n",
       "  1227,\n",
       "  34959,\n",
       "  15144,\n",
       "  11938,\n",
       "  2889,\n",
       "  20318,\n",
       "  3701,\n",
       "  13944,\n",
       "  13572,\n",
       "  6910,\n",
       "  34682,\n",
       "  1708,\n",
       "  1649,\n",
       "  29045,\n",
       "  324,\n",
       "  17849,\n",
       "  16969,\n",
       "  5567,\n",
       "  44320,\n",
       "  648,\n",
       "  19,\n",
       "  46981,\n",
       "  3547,\n",
       "  13630,\n",
       "  3384,\n",
       "  489,\n",
       "  2544,\n",
       "  7752,\n",
       "  26060,\n",
       "  31781,\n",
       "  17486,\n",
       "  1253,\n",
       "  6185,\n",
       "  340,\n",
       "  42631,\n",
       "  766,\n",
       "  2714,\n",
       "  13,\n",
       "  2714,\n",
       "  2714,\n",
       "  5978,\n",
       "  412,\n",
       "  15064,\n",
       "  14600,\n",
       "  648,\n",
       "  3182,\n",
       "  6592,\n",
       "  494,\n",
       "  19114,\n",
       "  12413,\n",
       "  44504,\n",
       "  23457,\n",
       "  18070,\n",
       "  11385,\n",
       "  7882,\n",
       "  25,\n",
       "  39304,\n",
       "  63,\n",
       "  38115]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aeb0c71f-a2eb-4f2c-a946-3bcc4c4817e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "max_seq_length = context_length\n",
    "out_model_path = \"mygpt_unigram8\"\n",
    "train_epoches = 5\n",
    "batch_size = 15\n",
    "\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)\n",
    "\n",
    "\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "        output_dir=out_model_path,\n",
    "        overwrite_output_dir=True,\n",
    "        num_train_epochs=train_epoches,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        save_steps=2000,\n",
    "        save_total_limit=2,\n",
    "        prediction_loss_only=True,\n",
    "        #fp16=True, v100没法用\n",
    "    )\n",
    "\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"valid\"],\n",
    "    data_collator=data_collator,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "39674ddf-68ee-42dc-af05-485311c8f19d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='90225' max='90225' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [90225/90225 19:28:25, Epoch 5/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>10.446400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>10.169600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>9.912500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>9.747800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>9.652600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>9.570400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>9.495000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>9.442700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>9.408800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>9.362200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5500</td>\n",
       "      <td>9.332900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>9.316800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6500</td>\n",
       "      <td>9.272800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>9.243600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7500</td>\n",
       "      <td>9.223000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>9.222200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8500</td>\n",
       "      <td>9.195800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9000</td>\n",
       "      <td>9.189200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9500</td>\n",
       "      <td>9.168000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10000</td>\n",
       "      <td>9.142100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10500</td>\n",
       "      <td>9.137300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11000</td>\n",
       "      <td>9.108700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11500</td>\n",
       "      <td>9.112100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12000</td>\n",
       "      <td>9.078000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12500</td>\n",
       "      <td>9.095300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13000</td>\n",
       "      <td>9.090500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13500</td>\n",
       "      <td>9.064800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14000</td>\n",
       "      <td>9.051400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14500</td>\n",
       "      <td>9.025000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15000</td>\n",
       "      <td>9.045800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15500</td>\n",
       "      <td>9.015600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16000</td>\n",
       "      <td>9.010200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16500</td>\n",
       "      <td>9.018700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17000</td>\n",
       "      <td>8.988300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17500</td>\n",
       "      <td>8.981500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18000</td>\n",
       "      <td>8.970200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18500</td>\n",
       "      <td>8.924200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19000</td>\n",
       "      <td>8.947000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19500</td>\n",
       "      <td>8.942300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20000</td>\n",
       "      <td>8.916200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20500</td>\n",
       "      <td>8.937500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21000</td>\n",
       "      <td>8.901200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21500</td>\n",
       "      <td>8.893200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22000</td>\n",
       "      <td>8.891000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22500</td>\n",
       "      <td>8.889300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23000</td>\n",
       "      <td>8.886400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23500</td>\n",
       "      <td>8.863900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24000</td>\n",
       "      <td>8.857500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24500</td>\n",
       "      <td>8.874600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25000</td>\n",
       "      <td>8.870900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25500</td>\n",
       "      <td>8.867900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26000</td>\n",
       "      <td>8.847500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26500</td>\n",
       "      <td>8.857000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27000</td>\n",
       "      <td>8.834400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27500</td>\n",
       "      <td>8.870500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28000</td>\n",
       "      <td>8.836100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28500</td>\n",
       "      <td>8.809800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29000</td>\n",
       "      <td>8.812900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29500</td>\n",
       "      <td>8.811800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30000</td>\n",
       "      <td>8.822700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30500</td>\n",
       "      <td>8.811600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31000</td>\n",
       "      <td>8.798500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31500</td>\n",
       "      <td>8.790600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32000</td>\n",
       "      <td>8.793000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32500</td>\n",
       "      <td>8.804900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33000</td>\n",
       "      <td>8.783200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33500</td>\n",
       "      <td>8.799200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34000</td>\n",
       "      <td>8.763100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34500</td>\n",
       "      <td>8.782200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35000</td>\n",
       "      <td>8.771500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35500</td>\n",
       "      <td>8.765600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36000</td>\n",
       "      <td>8.748600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36500</td>\n",
       "      <td>8.756800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37000</td>\n",
       "      <td>8.720000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37500</td>\n",
       "      <td>8.740000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38000</td>\n",
       "      <td>8.721900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38500</td>\n",
       "      <td>8.721800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39000</td>\n",
       "      <td>8.739300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39500</td>\n",
       "      <td>8.740700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40000</td>\n",
       "      <td>8.728100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40500</td>\n",
       "      <td>8.714000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41000</td>\n",
       "      <td>8.719900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41500</td>\n",
       "      <td>8.721000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42000</td>\n",
       "      <td>8.708300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42500</td>\n",
       "      <td>8.722500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43000</td>\n",
       "      <td>8.688500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43500</td>\n",
       "      <td>8.695000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44000</td>\n",
       "      <td>8.679700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44500</td>\n",
       "      <td>8.678200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45000</td>\n",
       "      <td>8.683900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45500</td>\n",
       "      <td>8.677600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46000</td>\n",
       "      <td>8.704300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46500</td>\n",
       "      <td>8.666600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47000</td>\n",
       "      <td>8.679900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47500</td>\n",
       "      <td>8.679200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48000</td>\n",
       "      <td>8.672400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48500</td>\n",
       "      <td>8.671800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49000</td>\n",
       "      <td>8.668700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49500</td>\n",
       "      <td>8.641000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50000</td>\n",
       "      <td>8.653500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50500</td>\n",
       "      <td>8.671200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51000</td>\n",
       "      <td>8.640100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51500</td>\n",
       "      <td>8.661500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52000</td>\n",
       "      <td>8.652700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52500</td>\n",
       "      <td>8.653900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53000</td>\n",
       "      <td>8.639800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53500</td>\n",
       "      <td>8.643500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54000</td>\n",
       "      <td>8.671100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54500</td>\n",
       "      <td>8.653400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55000</td>\n",
       "      <td>8.624400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55500</td>\n",
       "      <td>8.615600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56000</td>\n",
       "      <td>8.610500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56500</td>\n",
       "      <td>8.622600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57000</td>\n",
       "      <td>8.620500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57500</td>\n",
       "      <td>8.629900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58000</td>\n",
       "      <td>8.611600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58500</td>\n",
       "      <td>8.610600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59000</td>\n",
       "      <td>8.614300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59500</td>\n",
       "      <td>8.600400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60000</td>\n",
       "      <td>8.611100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60500</td>\n",
       "      <td>8.618500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61000</td>\n",
       "      <td>8.593100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>61500</td>\n",
       "      <td>8.619700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62000</td>\n",
       "      <td>8.591000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>62500</td>\n",
       "      <td>8.609800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63000</td>\n",
       "      <td>8.602600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>63500</td>\n",
       "      <td>8.594800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64000</td>\n",
       "      <td>8.586300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64500</td>\n",
       "      <td>8.579300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65000</td>\n",
       "      <td>8.603700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65500</td>\n",
       "      <td>8.606800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66000</td>\n",
       "      <td>8.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>66500</td>\n",
       "      <td>8.575400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67000</td>\n",
       "      <td>8.586300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67500</td>\n",
       "      <td>8.591500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68000</td>\n",
       "      <td>8.588200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68500</td>\n",
       "      <td>8.603700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69000</td>\n",
       "      <td>8.617400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>69500</td>\n",
       "      <td>8.573700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70000</td>\n",
       "      <td>8.557200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70500</td>\n",
       "      <td>8.556700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71000</td>\n",
       "      <td>8.608000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>71500</td>\n",
       "      <td>8.591100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72000</td>\n",
       "      <td>8.595100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>72500</td>\n",
       "      <td>8.560200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73000</td>\n",
       "      <td>8.557900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73500</td>\n",
       "      <td>8.556700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74000</td>\n",
       "      <td>8.564400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74500</td>\n",
       "      <td>8.522200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75000</td>\n",
       "      <td>8.542700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75500</td>\n",
       "      <td>8.558100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76000</td>\n",
       "      <td>8.547500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76500</td>\n",
       "      <td>8.570100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77000</td>\n",
       "      <td>8.551800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77500</td>\n",
       "      <td>8.571300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78000</td>\n",
       "      <td>8.543100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>78500</td>\n",
       "      <td>8.558200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>79000</td>\n",
       "      <td>8.579600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>79500</td>\n",
       "      <td>8.519500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80000</td>\n",
       "      <td>8.552500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80500</td>\n",
       "      <td>8.549200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81000</td>\n",
       "      <td>8.542500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81500</td>\n",
       "      <td>8.571600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>82000</td>\n",
       "      <td>8.559400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>82500</td>\n",
       "      <td>8.551800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>83000</td>\n",
       "      <td>8.542100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>83500</td>\n",
       "      <td>8.531600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>84000</td>\n",
       "      <td>8.567200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>84500</td>\n",
       "      <td>8.540400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85000</td>\n",
       "      <td>8.565200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>85500</td>\n",
       "      <td>8.536900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>86000</td>\n",
       "      <td>8.531900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>86500</td>\n",
       "      <td>8.526100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>87000</td>\n",
       "      <td>8.537400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>87500</td>\n",
       "      <td>8.546300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>88000</td>\n",
       "      <td>8.558500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>88500</td>\n",
       "      <td>8.545300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>89000</td>\n",
       "      <td>8.530900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>89500</td>\n",
       "      <td>8.542900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90000</td>\n",
       "      <td>8.537500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "IOPub message rate exceeded.\n",
      "The Jupyter server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--ServerApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "ServerApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "ServerApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/liming/anaconda3/envs/pytorch/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    }
   ],
   "source": [
    "trainer.train()\n",
    "trainer.save_model(out_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "077b7101-c2fe-489b-bf97-a90f1d8e4c0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2a1bb64-d798-49c5-867f-79f1fbe16189",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
